{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# 请自行在log(x+1)特征和tf_idf特征上尝试，并比较不同特征的结果，\n",
    "# path to where the data lies\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-5.551115e-17</td>\n",
       "      <td>-4.625929e-18</td>\n",
       "      <td>-1.040834e-17</td>\n",
       "      <td>-1.457168e-16</td>\n",
       "      <td>1.619075e-17</td>\n",
       "      <td>2.613650e-16</td>\n",
       "      <td>2.498002e-16</td>\n",
       "      <td>1.815677e-16</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.141852e+00</td>\n",
       "      <td>-2.552931e+00</td>\n",
       "      <td>-4.002619e+00</td>\n",
       "      <td>-2.516429e+00</td>\n",
       "      <td>-1.467353e+00</td>\n",
       "      <td>-2.074783e+00</td>\n",
       "      <td>-1.189553e+00</td>\n",
       "      <td>-1.041549e+00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-8.448851e-01</td>\n",
       "      <td>-7.201630e-01</td>\n",
       "      <td>-6.937615e-01</td>\n",
       "      <td>-4.675972e-01</td>\n",
       "      <td>-2.220849e-01</td>\n",
       "      <td>-7.212087e-01</td>\n",
       "      <td>-6.889685e-01</td>\n",
       "      <td>-7.862862e-01</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-2.509521e-01</td>\n",
       "      <td>-1.530732e-01</td>\n",
       "      <td>-3.198993e-02</td>\n",
       "      <td>-1.230129e-02</td>\n",
       "      <td>-1.815412e-01</td>\n",
       "      <td>-2.258989e-02</td>\n",
       "      <td>-3.001282e-01</td>\n",
       "      <td>-3.608474e-01</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.399473e-01</td>\n",
       "      <td>6.112653e-01</td>\n",
       "      <td>6.297816e-01</td>\n",
       "      <td>3.291706e-01</td>\n",
       "      <td>-1.554775e-01</td>\n",
       "      <td>6.032562e-01</td>\n",
       "      <td>4.662269e-01</td>\n",
       "      <td>6.602056e-01</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.906578e+00</td>\n",
       "      <td>2.542658e+00</td>\n",
       "      <td>4.104082e+00</td>\n",
       "      <td>7.955377e+00</td>\n",
       "      <td>8.170442e+00</td>\n",
       "      <td>5.042397e+00</td>\n",
       "      <td>5.883565e+00</td>\n",
       "      <td>4.063716e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "count  7.680000e+02                  7.680000e+02    7.680000e+02   \n",
       "mean  -5.551115e-17                 -4.625929e-18   -1.040834e-17   \n",
       "std    1.000652e+00                  1.000652e+00    1.000652e+00   \n",
       "min   -1.141852e+00                 -2.552931e+00   -4.002619e+00   \n",
       "25%   -8.448851e-01                 -7.201630e-01   -6.937615e-01   \n",
       "50%   -2.509521e-01                 -1.530732e-01   -3.198993e-02   \n",
       "75%    6.399473e-01                  6.112653e-01    6.297816e-01   \n",
       "max    3.906578e+00                  2.542658e+00    4.104082e+00   \n",
       "\n",
       "       Triceps_skin_fold_thickness  serum_insulin           BMI  \\\n",
       "count                 7.680000e+02   7.680000e+02  7.680000e+02   \n",
       "mean                 -1.457168e-16   1.619075e-17  2.613650e-16   \n",
       "std                   1.000652e+00   1.000652e+00  1.000652e+00   \n",
       "min                  -2.516429e+00  -1.467353e+00 -2.074783e+00   \n",
       "25%                  -4.675972e-01  -2.220849e-01 -7.212087e-01   \n",
       "50%                  -1.230129e-02  -1.815412e-01 -2.258989e-02   \n",
       "75%                   3.291706e-01  -1.554775e-01  6.032562e-01   \n",
       "max                   7.955377e+00   8.170442e+00  5.042397e+00   \n",
       "\n",
       "       Diabetes_pedigree_function           Age      Target  \n",
       "count                7.680000e+02  7.680000e+02  768.000000  \n",
       "mean                 2.498002e-16  1.815677e-16    0.348958  \n",
       "std                  1.000652e+00  1.000652e+00    0.476951  \n",
       "min                 -1.189553e+00 -1.041549e+00    0.000000  \n",
       "25%                 -6.889685e-01 -7.862862e-01    0.000000  \n",
       "50%                 -3.001282e-01 -3.608474e-01    0.000000  \n",
       "75%                  4.662269e-01  6.602056e-01    1.000000  \n",
       "max                  5.883565e+00  4.063716e+00    1.000000  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression + GridSearchCV\n",
    "### 评价指标为 log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='auto',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='liblinear',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'C': [0.1, 1, 10, 100, 1000], 'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4, return_train_score=True)#对每一组参数做5折交叉验证，所以有5重循环\n",
    "grid.fit(X_train,y_train) ## fit是对每一组参数做一个循环，多一组，时间就线性增长"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4759667474477391\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU5bnA8d8zmawQICRhDZsIVXYEWRRxgRa0CqIgIIpWLa0Xqq2tS7XXKljrUqu3t/YqYt1AEBeUIgpFRSsFBCmrKAKyhEVCgLAkZJvn/nFOwhAmyQyZySTk+frJZ855z3vOec5IzpP3LO8rqooxxhgTLE+0AzDGGFO7WOIwxhgTEkscxhhjQmKJwxhjTEgscRhjjAmJN9oBVIe0tDRt27ZttMMwxpha5csvv9yvqully+tE4mjbti0rV66MdhjGGFOriMj2QOV2qcoYY0xILHEYY4wJiSUOY4wxIbHEYYwxJiSWOIwxxoTEEocxxpiQWOIwxhgTEkscFXh5yXfMX7eHwmJftEMxxpgao068AHg6fD5l1oqdfL33CM0aJHB939aM6dOKJskJ0Q7NGGOiSurCQE69e/fW03lzvNinfPL1Pl5dtp3PNmURGyNc3qU5N13QhvNapyAiEYjWGGNqBhH5UlV7ly23FkcFYjzC4E5NGdypKVuzjjJ92Q7e/HInc9fsplPzBtx0QRuGdW9JYlxMtEM1xphqYy2OEOUWFPHuf3bz6tJtfL33CA0TYxnVK4Mb+7ehTWq9sOzDGGNqgvJaHBG9OS4iQ0XkGxHZLCL3VVBvpIioiPR258eJyGq/H5+I9HCXjRWRdSKyVkQ+FJG0SB5DWUlxXq7v25oP7ryI2T/rz4AOabz8721c8qfF3PzSF3zy9T58vjM/GRtj6q6ItThEJAbYBPwQyARWAGNV9asy9ZKB94E4YJKqriyzvCvwnqqeJSJeYDfQSVX3i8gTQK6qPlRRLOFscQTy/eHjvL58B69/sYOsI/m0bpzEjf3aMKp3Bo2S4iK2X2OMiaRotDj6AJtVdauqFgCzgOEB6k0BngCOl7OdscBMd1rcn3ri3JlugJNIoqppgwR+9cOOLLn3Mv53bE+aNUjgD/M30vfRj7j3rbWs35UT7RCNMSZsInlzvCWw028+E+jrX0FEegKtVHWeiPymnO2Mxk04qlooIrcD64BjwLfAxEAricgEYAJA69atq3AYwYvzeriqewuu6t6CjXsO8+rS7bz7n128sXInvdqkML5/Gy7v0pw4r70+Y4ypvSJ5Bgv0rGrpdTER8QBPA78udwMifXEuRa1352OB24GeQAtgLfDbQOuq6lRV7a2qvdPTTxnAKuLObd6AP17TlWX3D+K/r+zEgWMF3DlrNRc89jFPLfyGPTl51R6TMcaEQyQTRybQym8+g5MvKyUDXYDFIrIN6AfMLblB7hrDictUAD0AVHWLOjdnZgMXhD/08GmYGMutA9rx0V0X88otfejRqiF//WQzAx7/hNunf8m/t+ynLjzZZow5c0TyUtUKoIOItAN24SSB60sWqmoOUPpElIgsBn5TcnPcbZGMAgb6bXMX0ElE0lU1C+fG+8YIHkPYeDzCxR3TubhjOjsP5DJ9+XbeWLGTD9bvpUOT+ozv34YR52VQP95erTHG1GwRa3GoahEwCViAc3KfraobRGSyiAwLYhMDgUxV3eq3zd3Aw8BnIrIWpwXyaPijj6xWjZP47eXnsuy3g3hyZDcSYmP47/c20O/Rj/j9e+vZvO9otEM0xphy2QuANYCqsnrnIV5dup331+6hoNjHhWenMr5/Wwad0wRvjN1MN8ZUv/Iex7XEUcPsP5rPGyt2MmPZdnbnHKdFwwTG9WvDmPNbkVo/PtrhGWPqEEsctSRxlCgq9rFo4z5eW7aNJZuziYvxcGW35tzYvw09WjWyDhaNMRFnnRzWMt4YD0O7NGNol2Zs3neE15Zu5+1Vu3jnP7voltGQG/u14aruLUiItQ4WjTHVy1octcjR/CLmrMrk1aXb+XbfURolxTL6/Fbc0LcNrRonRTs8Y8wZxi5VnQGJo4SqsnRrNq8t3c7Cr77Hp8qgc5owvn9bBpydhsdjl7GMMVVnl6rOICLCBe3TuKB9Gnty8nh9+Q5mfrGDRRu/oF1aPW7o14aRvTJomBgb7VCNMWcga3GcIfKLivlw/V5e+fc2Vu04RGJsDFf3bMn4/m04t3mDaIdnjKmF7FLVGZ44/K3flcOrS7fx3urd5Bf56NO2MeMvaMOQzs2ItXdCjDFBssRxGolj+lfTyS/OJz0pnfRE9ycpnQZxDWrF47CHcguYvXIn05ftYMeBXJokxzO2T2uu79uapg0Soh2eMaaGs8RxGonj2rnXsungplPK4zxxJ5JJUjppiWk0SWpCWmJaaVl6YjqN4mvG+xY+n/LppixeWbqNxd9k4fUIQ7s0Y3z/tpzfNqVGxGiMqXkscZzmparcwlyy8rLIys1if95+9uXucz7z9rE/d3/psiOFR05Z1+vxlrZU0hLTShNKaZJx51MSUvBI9VxC2rb/GNOXbWf2yp0cPl7EOc2SGd+/LVf3bEFSnD0rYYw5wRJHhO9x5BXlsT9vP1m5WWTlnZxkSsqy8rLIyT91NECveGmc2JgmiU1ISzq51eI/3TihMTGe8Lzwl1dQzHurd/HK0u1s3HOY5AQvo3q14sb+bWiXVi8s+zDG1G6WOGrIzfH84vyTk0mZlkxJ2cH8g6es6xEPqQmpJ18aC5BgUhNT8XqCaz2oKl9uP8irS7czf90einzKwI7pjO/XhkvPaUKMvRMSMlWlwFdAbmEuuUW55BXmkVvkTOcW5pJX5MwfPn6MYwXHKVYfPlWK1Yeq4vM50z58+NSHz4fz6dZzfty66sOnOHV9PnwoWlKmPhTF53M/S9aD0vWd8hPbUtSZdj9Vfficg8LHiXX814UT+8JvWemn+x+qOFsGddcru+ykMr9PtGS6wm++giWne56raL3gtnlqreqN5fPrF9Mw4fReELbEUUMSR7AKiwvJPp7Nvtx9Tgsm1708VibpHDh+4JRfCkFISUgpTS6B7r+UXD6LjTnxrse+I8eZ9cVOZizfzveH88lISeSGfm0Y3bsVKfXiqvsrqBaFxYXOyb0or/RE739yLzsdqG5uUR5HC46V1s0vzsM93VaJqlA6kGbpdEkiF7eME+UqKGXL/KZL5z2nbEMJZl+ClJQLiIp7f8zj1jqxXEROzPtNO/MeSrcmnhNLpKJPj982y1fR/bqK1qz4Pp8EmApmfxXMVfj3WAX78y+RCpb5mTnyDyTFnl4HqZY4alniCFaRr4jsvOzSVsspl8ncBJN9PLv0r0N/KfEppCWlOZfJ3BZM44Q0du+P5fOvC1i/U4mjEVd1a81N/dvSNaNhFI7SOU7/E/ZJJ2+/v+pLyis72ZcsL/IVBR2DEEMMCYjGo744fMWxFBfHUVwUi/riUI0Dn7MMXxzqiyOGeBK99UiOSyI5PomG8fVplFCfxknJpNarT4O4RGJiPMSIhxgRYsSDx+PBGyN45MRPjMc5ScWI4PHglvnXgRiPc5KN8Tj1xC0ru75Tj5DW90hlJ1dzJrLEcYYmjmAV+4o5mH/wpPsuZW/wZ+VlkZ2XTZGeejLV4iR8hcnU96bwg/SWdG/emmb1mpzcgklKIz4mPuSTt//y0ss6ZZYX+AqCPtYYiSHRm0h8TCKxngS8koC39ITvnOSLi+MoLIyloNDL8Xwvefkx5BV4/U768aUnf/XF4dE4GiUl0SgxlkZJsTRKinM+E+NISYqlUb04GiXGklJS7tapFxdjJ1xTa0WlyxERGQr8DxADTFPVx8qpNxJ4EzhfVVeKyDjgbr8q3YDzVHW1iMQBfwUuAXzAA6r6dgQP44wQ44khLTGNtMS0Cuv51MfB4wdPabXsOrKXtXt2su3QXlbt+5LVBxaDFFcpJkFIik0i0ZtIkjeJpNgkkrxJNIhrQLN6zUj0JhLrSUR8ceCe9IuK4igu8lJQGEt+QSx5+R5y870cyfNwNDeGQ8fgcJ5ySMs/WTdI8JLinuhbJsXRqIHfCT8xlpR6cTQ8KQnEkRzvtT7AjHFFLHGISAzwLM644JnAChGZq6pflamXDNwBLC8pU9UZwAx3eVfgPVVd7S5+ANinqh3dcckbR+oY6iKPeEhNTCU1MZUfNP7BKct9PmXJlv288u9tfLzpOyT2CL3axdCng5eG9fMo9BWWJoFEb2LpZwwJFBfFUlAYy/H8GI4XejmaJ+TkFXEot4BDuYUcPFJITl4Bu3ILOZTrTBcWl98irh/vdU7w9Zy//Ns2P/lk38hd1rCkVZDkJAS74W9M1USyxdEH2FwyZriIzAKGA1+VqTcFeAL4TTnbGQvM9Ju/BTgHQJ1HM/aHMWZTCY9HuKhDOhd1SCfzYGdmLN/BGyt28sXGAtqnZ9C7TWNy8go5mFtQ+nkoN4f8olOfEiuREOshJenEX/kdm9b3O9n7J4E491KRUzfOa92nGBMNkUwcLYGdfvOZQF//CiLSE2ilqvNEpLzEMRon4SAijdyyKSJyCbAFmKSq35ddSUQmABMAWrduXYXDMOXJSEni3qHncOegDry/dg/Tl2/n42/2OSf8xDhaN06ie0ajk+4JpCS5LQC3ldAoKdYGozKmlolk4gh0PaD0uoN7melp4OZyNyDSF8hV1fVukRfIAJao6l0ichfwJ+DGU3akOhWYCs7N8dM8BhOEhNgYru2VwbW9MqIdijGmGkSyrZ8JtPKbzwB2+80nA12AxSKyDegHzBUR/zv4Yzj5MlU2kAvMceffBM4Lb9jGGGMqEsnEsQLoICLt3CehxgBzSxaqao6qpqlqW1VtCywDhqnqSihtkYwCZvmto8A/cJ6oAhjEqfdMjDHGRFDELlWpapGITAIW4DyO+3dV3SAik4GVqjq34i0wEMgsubnu517gNRF5BsgCfhLu2I0xxpTPXgA0xhgTUHkvANrzjMYYY0JiicMYY0xILHEYY4wJiSUOY4wxIbHEYYwxJiSWOIwxxoTEEocxxpiQWOIwxhgTEkscxhhjQmKJwxhjTEgscRhjjAmJJQ5jjDEhscRhjDEmJJY4jDHGhMQShzHGmJBY4jDGGBOSiCYOERkqIt+IyGYRua+CeiNFREvGGxeRcSKy2u/HJyI9yqwzV0TWRzJ+Y4wxp4pY4hCRGOBZ4HKgEzBWRDoFqJcM3AEsLylT1Rmq2kNVewA3AttUdbXfOtcARyMVuzHGmPJFssXRB9isqltVtQCYBQwPUG8K8ARwvJztjAVmlsyISH3gLuCR8IZrjDEmGJFMHC2BnX7zmW5ZKRHpCbRS1XkVbGc0fokDJ9E8BeRWtHMRmSAiK0VkZVZWVkiBG2OMKV8kE4cEKNPShSIe4Gng1+VuQKQvkKuq6935HsDZqjqnsp2r6lRV7a2qvdPT00MO3hhjTGCRTByZQCu/+Qxgt998MtAFWCwi24B+wNySG+SuMZzc2ugP9HLrfw50FJHFYY/cGGNMuSKZOFYAHUSknYjE4SSBuSULVTVHVdNUta2qtgWWAcNUdSWUtkhG4dwbKVnn/1S1hVt/ALBJVS+J4DEYY4wpI2KJQ1WLgEnAAmAjMFtVN4jIZBEZFsQmBgKZqro1UjEaY4wJnahq5bVqud69e+vKlSujHYYxxtQqIvKlqvYuW25vjhtjjAmJJQ5jjDEhscRhjDEmJJY4jDHGhMQShzHGmJB4ox2AMcZEUmFhIZmZmRw/Xl53eCYhIYGMjAxiY2ODqm+JwxhzRsvMzCQ5OZm2bdsiEqgnpLpNVcnOziYzM5N27doFtY5dqjLGnNGOHz9OamqqJY1yiAipqakhtcgscRhjzniWNCoW6vdjicMYYyKsfv36pdNDhw6lUaNGXHnllQHrTpw4kR49etCpUycSExPp0aMHPXr04K233gppn6tWreLDDz+sUtzlsXscxhhTje6++25yc3N5/vnnAy5/9tlnAdi2bRtXXnklq1evDlivMqtWrWL9+vUMHTr0tGMtj7U4jDGmGg0aNIjk5OTTWvfbb79lyJAh9OrVi4EDB7Jp0yYAZs2aRZcuXejevTuXXnopeXl5TJ48mRkzZpxWa6Uy1uIwxtQZD/9jA1/tPhzWbXZq0YDfX9U5rNssz4QJE5g2bRrt27dnyZIlTJo0iYULF/Lwww+zePFimjZtyqFDh0hMTOTBBx9k/fr1PPPMM2GPwxKHMcbUAocOHWLZsmVce+21pWVFRUUAXHjhhYwfP55Ro0ZxzTXXRDwWSxzGmDqjuloGkaCqpKWlBbzn8cILL7B8+XLmzZtH9+7dWbt2bURjsXscxhhTC6SkpNC8eXPmzJkDgM/nY82aNQBs3bqVfv36MWXKFFJSUti1axfJyckcOXIkIrFENHGIyFAR+UZENovIfRXUGykiWjLeuIiME5HVfj8+EekhIkki8r6IfC0iG0TksUjGb4wx4XbRRRcxatQoPvroIzIyMliwYEHQ686aNYvnnnuO7t2707lzZ+bNmwfAr371K7p27UrXrl0ZPHgwXbp04bLLLmPNmjX07Nkz7DfHIzYCoIjEAJuAHwKZOGOQj1XVr8rUSwbeB+KASSVjjvst7wq8p6pniUgS0FdVP3HHMf8IeFRVP6goFhsB0Ji6a+PGjZx77rnRDqPGC/Q9RWMEwD7AZlXdqqoFwCxgeIB6U4AngPLedx8LzARQ1VxV/cSdLgBWARnhDtwYY0z5Ipk4WgI7/eYz3bJSItITaKWq8yrYzmjcxFFm3UbAVTitjlOIyAQRWSkiK7OyskKN3RhjTDkimTgCdX5Sel1MRDzA08Cvy92ASF8gV1XXlyn34iSTv6jq1kDrqupUVe2tqr3T09NPJ35jjDEBhJw4RMQjIg2CqJoJtPKbzwB2+80nA12AxSKyDegHzC25Qe4aQ4DWBjAV+FZVw/9mizHGmAoFlThE5HURaSAi9YCvgG9E5O5KVlsBdBCRdu6N7DHA3JKFqpqjqmmq2lZV2wLLgGElN8fdFskonHsj/rE8AjQEfhnUERpjjAmrYFscnVT1MHA1MB9oDdxY0QqqWgRMAhYAG4HZqrpBRCaLyLAg9jkQyPS/FCUiGcADQCdglfuo7m1BHoMxxpgwCDZxxIpILE7ieE9VC/G7X1EeVZ2vqh1Vtb2q/sEte1BV5waoe4n/o7iqulhV+5Wpk6mqoqrnqmoP92dakMdgjDFRUd3dqs+ZM4cnn3yyynGXJ9guR54HtgFrgM9EpA0Q3p7CjDGmDghXt+pFRUV4vYFP4SNGjAhPsOUIqsWhqn9R1ZaqeoU6tgOXRjQyY4w5A1WlW/UBAwbwwAMPMHDgQP7617/y3nvv0bdvX3r27MmPfvQj9u3bB8C0adP45S+d28A33HADd955JxdccAFnnXVWaZclVRFUi0NE7gReAo4A04CewH3AwipHYIwx1eWD+2DvuvBus1lXuLz6ej86fPgwn332GQAHDx5k2LBhiAjPPfccTz31FI8//vgp6+zbt48lS5awbt06rrvuuiq3SIK9VHWLqv6PiAwB0oGf4CQSSxzGGFONxowZUzq9Y8cOrrvuOvbu3Ut+fj4dO3YMuM7VV1+NiNCtWzd27dpV5RiCTRwlL/NdAbykqmvERn83xtQ21dgyiJR69eqVTk+cOJH777+fK664gkWLFvHYY4GPLz4+vnQ6HP0TBvtU1ZcishAncSxwOyb0VXnvxhhjTltOTg4tW7ZEVXnllVeqbb/BtjhuBXoAW1U1V0RScS5XGWOMCcFFF13E119/zdGjR8nIyODFF19kyJAhp7Wthx56iBEjRpCRkUGfPn3Ys2dPmKMNLOhu1d2X9ga6s5+q6j8iFlWYWbfqxtRd1q16cMLerbo7YNKdON2NfAXcISJ/DEOsxhhjaplgL1VdAfRQVR+AiLwC/Af4baQCM8YYUzOF0jtuI7/phuEOxBhjTO0QbIvjj8B/ROQTnEdzB2KtDWOMqZOCShyqOlNEFgPn4ySOe1V1byQDM8YYUzNVmDhE5LwyRZnuZwsRaaGqqyITljHGmJqqshbHUxUsU+CyMMZijDFnpPr163P06FFWr17N7bffzuHDh4mJieGBBx5g9OjRJ9WdOHEiS5YsoaCggO+++44f/OAHAPzud79j5MiRQe9z1apV7Nu3j6FDh4b1WKCSxKGq1gOuMcaESVJSEq+++iodOnRg9+7d9OrViyFDhtCo0Ylnj4LtVr0yq1atYv369RFJHMG+x3FNgJ9BItKkkvWGisg3IrJZRO6roN5IEdGS8cZFZJw7ul/Jj09EerjLeonIOnebf7E+s4wxtUXHjh3p0KEDAC1atKBJkyZkZWUFvf63337LkCFD6NWrFwMHDmTTpk0AzJo1iy5dutC9e3cuvfRS8vLymDx5MjNmzAh5EKhghNLlSH/gE3f+EpwxwjuKyGRVfa3sCiISAzwL/BDn3sgKEZmrql+VqZcM3AEsLylT1RnADHd5V5xRB0vS7v8BE9z9zweGAh8EeRzGmDrs8S8e5+sDX4d1m+c0Pod7+9wb8npffPEFBQUFtG/fPuh1JkyYwLRp02jfvj1Llixh0qRJLFy4kIcffpjFixfTtGlTDh06RGJiIg8++CDr16/nmWeeCTm2ygSbOHzAuar6PYCINMU5gfcFPgNOSRxAH2BzyZjhIjILGI7z5rm/KcATwG/K2fdYYKa7jeZAA1Vd6s6/ijOcrSUOY0ytsWfPHm688UZeeeUVPJ7gXqc7dOgQy5Yt49prry0tKyoqAuDCCy9k/PjxjBo1imuuuSYiMfsLNnG0LUkarn1AR1U9ICKF5azTEtjpN5+Jk2hKiUhPoJWqzhOR8hLHaJyEU7LNTL9lmW7ZKURkAk7LhNatW5ez6Ups+QSSGkPz7qe3vjGmRjmdlkG4HT58mB//+Mc88sgj9OvXL+j1VJW0tLSA9zxeeOEFli9fzrx58+jevTtr164NZ8inCPbN8X+JyDwRuUlEbgLm4ow9Xg84VM46ge49lPaoKCIe4Gng1+XtVET6Armquj6YbZ5UqDpVVXurau/09PTydlG+ogKY90t4fTTkZFZe3xhjKlFQUMCIESNKWwehSElJoXnz5qVDv/p8PtasWQPA1q1b6devH1OmTCElJYVdu3aRnJzMkSNHwn4MEHzimIgz4l8PnGFjXwEmquqxCp68ygRa+c1nALv95pOBLsBiEdkG9APmltwgd43BvUzlt82MCrYZPt44GPM65B+FGdfB8cMR2Y0xpu6YPXs2n332GS+//DI9evSgR48eIT01NWvWLJ577jm6d+9O586dmTdvHgC/+tWv6Nq1K127dmXw4MF06dKFyy67jDVr1tCzZ8+w3xwPpVv1pjj3LRT4QlX3VVLfC2wCBgG7gBXA9aq6oZz6i4HfqOpKd94D7AAGltwncctXAL/AuZk+H/hfVZ1fUSxV6lZ9y8cwYxS0GwjXz4aY2NPbjjEmKqxb9eBEolv164AvgJHAdcByEanwTRRVLQImAQuAjcBsVd0gIpPdsT0qMxDI9E8artuBacBmYAuRvjHe/jK48mkngbx/F4Rh2EVjjKnNgr05/gBwfkkrQ0TSgUVAhe0ftyUwv0zZg+XUvaTM/GKcy1dl663EucRVfc4bDwe3wb+egpR2cNFd1bp7Y4ypSYJNHJ4yl6ayCa1L9trv0t85yeOjhyGlDXS5ttJVjDHmTBRs4vhQRBZw4kb1aMq0JM54Hg8M/xsc3g1zbocGLaF18I/SGWPMmSKoVoOq3g1MBboB3YGpqhr9B6KrW2yC86RVwwyYORayt0Q7ImOMqXZBX25S1bdV9S5V/ZWqzolkUDVaUmMY9yaIwIyRcCw72hEZY0y1qjBxiMgRETkc4OeIiNTdFxtS28OYmZCzC2ZdD4XHox2RMaYGq1+/PgCrV6+mf//+dO7cmW7duvHGG2+cUnfixIn06NGDTp06kZiYWPq+RyjvYsyZM4cnn3wybPGXFfR7HLVZld7jqMiGOfDmzdD5Grj2Rec+iDGmRqkJ73GUjMexadMmROSkbtU3btx4UrfqJUq6VV+/fn2ALTr9VHm9wd6mrlwo73GEb691UecRcHA7LPo9pLSFwb+PdkTGmBqsY8eOpdP+3aoHShyBDBgwgIsvvph//etfXHPNNbRr145HH32UgoIC0tPTmT59Ok2aNGHatGmlPePecMMNpKamsmLFCvbu3ctTTz3FiBEjqnQcljiq6sI74eB38Pmfncd0e90c7YiMMeXY++ij5G8Mb7fq8eeeQ7P77w95vdPpVh2cThI/++wzAA4ePMiwYcMQEZ577jmeeuopHn/88VPW2bdvH0uWLGHdunVcd911ljiiTgSueMrpCHHeXdCwFZw9KNpRGWNqsNPpVr3EmDFjSqd37NjBddddx969e8nPzz+pRePv6quvRkTo1q0bu3btqlLsYIkjPGK8MPIleOlymH0T3PIhNKvel9uNMZU7nZZBuJ1ut+ol6tWrVzo9ceJE7r//fq644goWLVrEY489FnCd+Pj40ulw3Ne2u7nhktDA6QQxvj68fh0c3hPtiIwxNUxVulUPJCcnh5YtW6KqvPLKK2GIMDiWOMKpYUsneRzPgddHQX5k+sI3xtROVe1WvayHHnqIESNGcPHFF9O0adMwRloxexw3Er79pzMA1NmDnPc9YuyKoDHRUhMex60Nwt6tuglRhx/CFU/Ctwvhg3usK3ZjzBnF/hSOlPNvdXrT/fdfoHE7uOAX0Y7IGGPCwhJHJA1+GA5th4X/DY1aQ6fh0Y7IGGOqLKKXqkRkqIh8IyKbReS+CuqNFBH1H29cRLqJyFIR2SAi60QkwS0f686vFZEPRSQtksdQJR4PjHgeMs6HdybAzhXRjsiYOqku3MutilC/n4glDhGJAZ4FLgc6AWNFpFOAesnAHThjiJeUeYHpwM9VtTNwCVDolv8PcKmqdgPW4gxPW3PFJsLYmZDcDGaOgQPfRTsiY+qUhIQEsrOzLXmUQ1XJzs4mISEh6HUieamqD7C5ZMxwEZkFDAe+KlNvCvAE8Bu/sh8Ba1V1DYCqZrvbiAUEqCci2aePzwcAABhGSURBVEADnLHHa7Z6aTDuLZg2GGaMglsXOt2zG2MiLiMjg8zMTLKysqIdSo2VkJBARkZG0PUjmThaAjv95jOBvv4VRKQn0EpV54mIf+LoCKg76mA6MEtVn1DVQhG5HVgHHAO+BSYG2rmITAAmALRu3TpMh1QFaR2cQaBeuxreuBFufAe88ZWvZ4ypktjYWNq1axftMM4okbzHIQHKStuKIuIBngZ+HaCeFxgAjHM/R4jIILfFcTvQE2iBc6nqt4F2rqpTVbW3qvZOT0+v0oGETdsLneFnt38Oc39hj+kaY2qlSLY4MoFWfvMZwG6/+WSgC7BYRACaAXNFZJi77qequh9AROYD5wGHAVR1i1s+Gyj3pnuN1G0UHNoGHz/idMV+afT7zjHGmFBEssWxAuggIu1EJA4YA8wtWaiqOaqapqptVbUtsAwYpqorgQVANxFJcm+IX4xzb2QX0ElESpoQPwQ2RvAYIuOi30DPG+DTx+E/M6IdjTHGhCRiLQ5VLRKRSThJIAb4u6puEJHJwEpVnVvBugdF5M84yUeB+ar6PoCIPAx8JiKFwHbg5kgdQ8SIwJXPOF2x/+MOp4+rsy6JdlTGGBMU66sqmo7nwItD4PBuuHUBNLH+dIwxNYf1VVUTJTSEcbMhNsF5TPfI99GOyBhjKmWJI9oatYbr34DcbJg5GgqORTsiY4ypkCWOmqBFTxj5d9izBt6+DXzF0Y7IGGPKZYmjpvjB5TD0MfhmPix4INrRGGNMuax33Jqk78+crtiX/c15x6Pfz6MdkTHGnMISR03zo0fg0A748D5o1ArO+XG0IzLGmJPYpaqaxhMD17zg3Pd4+zbYtSraERljzEkscdREcUnOk1b10pyxyw/tiHZExhhTyhJHTVW/CVz/JhTlO+945B2KdkTGGANY4qjZmpwDo1+D7C0w+0YoKoh2RMYYY4mjxjvrYhj2F/juM5j3S+uK3RgTdfZUVW3Q43o4uB0+fcx5TPfie6IdkTGmDrPEUVtccp/zjscnf4BGbaD76GhHZIypoyxx1BYiMOx/4fAueG+i0xV72wHRjsoYUwfZPY7axBvn3Cxv3A5mjYOsTdGOyBhTB1niqG0SU2DcmxATCzNGwtGsaEdkjKljIpo4RGSoiHwjIptFpNyxwUVkpIioiPT2K+smIktFZIOIrBORBLc8TkSmisgmEflaRK6N5DHUSCltYewsOPo9zBwDhXnRjsgYU4dELHGISAzwLHA50AkYKyKdAtRLBu4AlvuVeYHpwM9VtTNwCVDoLn4A2KeqHd3tfhqpY6jRMno7XZPs+hLemQA+X7QjMsbUEZFscfQBNqvqVlUtAGYBwwPUmwI8ARz3K/sRsFZV1wCoaraqlgxScQvwR7fcp6r7I3UANV6nYU6niBvnwqIHox2NMaaOiGTiaAns9JvPdMtKiUhPoJWqziuzbkdARWSBiKwSkXvc+o3c5VPc8jdFpGmgnYvIBBFZKSIrs7LO4PsA/SfC+T+Ff/8vrJgW7WiMMXVAJBOHBCgrfe1ZRDzA08CvA9TzAgOAce7nCBEZ5JZnAEtU9TxgKfCnQDtX1amq2ltVe6enp1fpQGo0EWcAqI5DYf7dsGlhtCMyxpzhIpk4MoFWfvMZwG6/+WSgC7BYRLYB/YC57g3yTOBTVd2vqrnAfOA8IBvIBea423jTLa/bYrxw7YvQtAu8ebMzBK0xxkRIJBPHCqCDiLQTkThgDDC3ZKGq5qhqmqq2VdW2wDJgmKquBBYA3UQkyb1RfjHwlaoq8A+cm+UAg4CvInUAh+a8y7EvvkBrQ/9Q8fXh+tnO47qvj4aczGhHZIw5Q0UscahqETAJJwlsBGar6gYRmSwiwypZ9yDwZ5zksxpYparvu4vvBR4SkbXAjQS+1FX1+IuLyX7uOXaMv4ntY6/nyMef1PwE0qA5jJsN+UdhxnVw/HC0IzLGnIGkxp8Mw6B37966cuXKkNfzHT/Oobff5sCLf6dw927iO3QgdcIEGlw+FPHW4N5atnzsjOHRbqDTComJjXZExphaSES+VNXeZcvtzfEKeBISaDxuHO0XfEiLxx9D1cfuu+9my+VXcHDWLHz5+dEOMbD2l8GVTzsJ5P27rCt2Y0xYWeIIgsTG0nD4cM6aO5eMZ/9KTEoKex96mM2DB5P94osUHz0W7RBPdd54uOjXsOpV+PzpaEdjjDmD2KWq06Cq5C5fTvbUqRz791I8DRqQMu56Gt94I97GjcO2nyrz+eCd22D92zDy79Cl7vXOYow5feVdqrLEUUV569aRPXUqR/65CElIoNGoUaTe8hNimzePyP5CVngcXrsadq2Cm+ZC637RjsgYU0tY4ohQ4iiRv2UL2S9MI2ee8xJ8w6uuIvWntxF/1lkR3W9Qcg/AtMGQdxBuWwSp7aMdkTGmFrDEEeHEUaJw1y6yX3qZQ2+9hebnk/zDH5I6YQKJXTpXy/7Llb3FSR6JjeDWRVAvNbrxGGNqPHuqqprEtmxJs989wNkfLSL1ZxM4tnQp20aOZMctt3JseRRfJkxt73TFnrMLZl3vXMIyxpjTYIkjQrypqTT55S85+5OPSf/1XRzftIkdN93E9jFjOfLxx2g0ukFv3RdGPAc7l8G7t1tX7MaY02KJI8Ji6tcn7ac/5exF/6TZ7x+kaP9+Mv9rIt8NH07O3LloUVH1BtTlGhj8MGx4Bz6eUr37NsacESxxVBNPQgIpY8c6LxM+8TgAu++5ly1DhnLg9dfxHa/GS0cX3gm9bobP/wxfvlx9+zXGnBHs5niUqM/H0cWLyX5+Knlr1hCTlkbjm8aTMnYsMfXrRz6A4iKYORq2fOKMYX72oMjv0xhTq9hTVTUscZRQVXK/WOG8TLhkCZ7kZOdlwvHjI/8y4fHD8NLlcHA73PIhNOsS2f0ZY2oVSxw1NHH4y1u/wX2Z8J9IfDyNRo50XiZs0SJyO83ZBdMGgXjgto+cHnaNMQZLHLUicZTI37qV7GkvkjPXGb6k4ZVXOi8Tto/Qi3t71jotj8bt4CcfQHxyZPZjjKlV7D2OWiT+rLNo8egfOHvhAlLGjuXwhx+y9cqryPzFL8hbty78O2zeDUa9DN9/BW/d4tz/MMaYcljiqMFiW7Sg2QP3c/bHH5H6859xbPkXbBt1HTtuuYVjy5aF92XCDj+EK56EbxfCB/dYV+zGmHJFNHGIyFAR+UZENovIfRXUGyki6o43XlLWTUSWisgGEVknIgll1pkrIusjGX9N4W3cmCZ33snZH39Ek7t/w/Fvv2XHzT9h2+gxHPnoo/C9THj+rXDBHbDyRVj61/Bs0xhzxolY4hCRGOBZ4HKgEzBWRDoFqJcM3AEs9yvzAtOBn6tqZ5wxxgv9ll8DHI1U7DVVTP36pN56K2cvWkSzh35P8YEDZE6cxNZhw8h57z20sLDyjVRm8MPQaTgs/G/46r2qb88Yc8aJZIujD7BZVbeqagEwCxgeoN4U4AnA/w24HwFrVXUNgKpmq2oxgIjUB+4CHolg7DWaJz6elDFjaP/hB7R48klEPOy+9z7nZcIZM6r2MqHHAyOeh4ze8M4E2LkifIEbY84IkUwcLYGdfvOZblkpEekJtFLVeWXW7QioiCwQkVUico/fsinAU0BuRTsXkQkislJEVmZlZZ32QdRk4vXS8Koraffeu2T87W94mzTh+ymPsHnQYPY/P5XiI0dOb8OxiU6HiMnNYOYYOPBdeAM3xtRqkUwcEqCs9I6riHiAp4FfB6jnBQYA49zPESIySER6AGer6pzKdq6qU1W1t6r2Tk9PP60DqC3E4yH5sktpM/N1Wr/6CgnnnkvW00+z+dLL2PfnpynKzg59o/XSYNxb4CuCGaOcMT2MMYbIJo5MoJXffAaw228+GegCLBaRbUA/YK57gzwT+FRV96tqLjAfOA/oD/Ry638OdBSRxRE8hlpFRKjXpw+tp71A27ffot6AAWS/8AKbLxvE3slTKNy1K7QNpnWAMa/Doe3wxo1QlB+ZwI0xtUrEXgB0b3BvAgYBu4AVwPWquqGc+ouB36jqShFJAT7CaW0UAB8CT6vq+3712wLzVLXSfjJq2wuA4ZS/9TuyX5xGztx/gCoNf/xj52XCs88OfiNr33TGLu822rn/IYEak8aYM021vwCoqkXAJGABsBGYraobRGSyiAyrZN2DwJ9xks1qYJV/0jDBiz+rHS3+8AfO/udCGo+7nsMLF7L1yqvYOWkSeWvXBreRbqPgst/B2jdg8R8jG7AxpsazLkfqmKKDBzn42mscmD4D3+HDJPXrR9rPJpDUrx9SUUtCFeZOgv9Mh+F/g57jqi9oY0xUWF9VljhOUnz0GIfeeIMDL79MUVYWCV27kjrhpyQPGoR4ymmIFhfCjJGw7XO44W0465LqDNkYU80scVjiCMiXn0/Ou++R/eKLFO7YQVz79qTedhsNr/wxEht76grHc+DFIXB4N9y6AJqcW/1BG2OqhXVyaALyxMeTMvo62s9/nxZP/Qnxetnz29+yecgQ53JWXt7JKyQ0hHGzITbBeUz3yPfRCdwYEzWWOAzgvkz44x/T7t05ZDz3f8Q2a873j7gvEz73PMWHD5+o3Kg1XP8G5GY7owgWHIte4MaYameJw5xEREi+5BLavj6DNtNfI6FzZ7KeeYbNlw1i31N/pmj/fqdii54w8u+wZw28fRv4iqMbuDGm2ljiMOVK6t2b1i9Mpd07b1PvogFkT5vG5kGD2Tt5MgWZu+AHl8PQx+Cb+bDggWiHa4ypJpY4TKUSOnUi4+mnaf/BfBoOu4qDb77FliFD2HXPPeQ3vgz6/Rcs/z9Y9ly0QzXGVAN7qsqErPD77znw0sscnD0bzc2l/mWXkdYuk8Rjn8LF90L9JhATBzGx4PH6TcdCjDtf4XSsX/1Y8MRE+5CNqZPscVxLHGFXdPAgB6fP4MD06fhyckhqHU9y2j5E1OniUtyeLt15Z9otF/ffnfvOoYhbr3S6pFxBPBATg8R4nQTj8Zt2fyQm9sS0xwveWLfcC14ncTnTJYkpDvHGls6LN86ddstj45zpmBPTeOOdZYGS20nTcW7CLJm2xGdqJ0scljgixnfsGAdnv8mBl16iaN++aIcTYUEkxVPquNPiFoiAiJsgq9jvV63sNqwuHnP0tFu0DE9S8mmtW17i8FY5KlPneerVI/UnN9P4xhsozskBVWc4W1Xw+cDnc8ZHL5n2KWhl0z6nE/6g6umJaXeZ/7QTj7t/DVCvuBiKi9CiQqcb+bLTxc4nvuIT8yXTvuIT65dM+3yor8hvutip7/OBr9jdv1/8UVHJH4w19u/JGhtYjeWMYBFeljhM2IjXizc1NdphGGMizJ6qMsYYExJLHMYYY0JiicMYY0xILHEYY4wJSUQTh4gMFZFvRGSziNxXQb2RIqLueOMlZd1EZKmIbBCRdSKSICJJIvK+iHztlj8WyfiNMcacKmKJQ0RigGeBy4FOwFgR6RSgXjJwB7Dcr8wLTAd+rqqdgUuAQnfxn1T1HKAncKGIXB6pYzDGGHOqSLY4+gCbVXWrqhYAs4DhAepNAZ4AjvuV/QhYq6prAFQ1W1WLVTVXVT9xywqAVUBGBI/BGGNMGZFMHC2BnX7zmW5ZKRHpCbRS1Xll1u0IqIgsEJFVInJP2Y2LSCPgKuCjQDsXkQkislJEVmZlZVXlOIwxxviJ5AuAgToGKH3tU5zXGZ8Gbg5QzwsMAM4HcoGP3FffP3LX9QIzgb+o6tZAO1fVqcBUt36WiGw/zeNIA/af5rqRZHGFxuIKjcUVmjM1rjaBCiOZODKBVn7zGcBuv/lkoAuwWJz+epoBc0VkmLvup6q6H0BE5gPncaJ1MRX4VlWfCSYQVU0/3YMQkZWB+mqJNosrNBZXaCyu0NS1uCJ5qWoF0EFE2olIHDAGmFuyUFVzVDVNVduqaltgGTBMVVcCC4Bu7lNUXuBi4CsAEXkEaAj8MoKxG2OMKUfEEoeqFgGTcJLARmC2qm4Qkcluq6KidQ8Cf8ZJPquBVar6vohkAA/gPKW1SkRWi8htkToGY4wxp4poJ4eqOh+YX6bswXLqXlJmfjrOI7n+ZZlUf6fKU6t5f8GyuEJjcYXG4gpNnYqrTozHYYwxJnysyxFjjDEhscRhjDEmJJY4yhCRUW4/WD7/vrMC1AuqH64wxtVYRP4pIt+6nynl1Ct2HxpYLSJzA9UJUzwVHr+IxIvIG+7y5SLSNlKxhBjXze57PSXfUcQfrhCRv4vIPhFZX85yEZG/uDGvFZHzIh1TkHFdIiI5ft9VwPuTEYirlYh8IiIb3d/FOwPUqfbvLMi4qv07c/vx+0JE1rhxPRygTnh/H1XVfvx+gHOBHwCLgd7l1IkBtgBnAXHAGqBThON6ArjPnb4PeLyceker4Tuq9PiB/wKec6fHAG/UkLhuBv5azf+mBuK8h7S+nOVXAB/gPPjRD1heQ+K6BJhXnd+Vu9/mwHnudDKwKcD/x2r/zoKMq9q/M/c7qO9Ox+L0+9evTJ2w/j5ai6MMVd2oqt9UUi3YfrjCaTjwijv9CnB1hPdXkWCO3z/et4BB4r7pGeW4qp2qfgYcqKDKcOBVdSwDGolI8xoQV1So6h5VXeVOH8F5nL9lmWrV/p0FGVe1c7+Do+5srPtT9qmnsP4+WuI4PZX2wxUBTVV1Dzj/gIEm5dRLcPvoWiYikUouwRx/aR113unJASI9IHmw/1+udS9vvCUirQIsr27R+PcUrP7uJZAPRKRzde/cvaTSE7/es11R/c4qiAui8J2JSIyIrAb2Af9U1XK/r3D8Pkb0PY6aSkQW4XRxUtYDqvpeMJsIUFbl55oriiuEzbRW1d0ichbwsYisU9UtVY2tjGCOPyLfUSWC2ec/gJmqmi8iP8f5K+yyCMdVmWh8V8FYBbRR1aMicgXwLtChunYuIvWBt4FfqurhsosDrFIt31klcUXlO1PVYqCHOJ2/zhGRLqrqf+8qrN9XnUwcqjq4ipuorB+u01JRXCLyvYg0V9U9bpN8Xznb2O1+bhWRxTh/FYU7cQRz/CV1MsXpNqYhkb8sUmlcqprtN/sC8HiEYwpGRP49VZX/SVFV54vI30QkTd0+5CJJRGJxTs4zVPWdAFWi8p1VFlc0vzN3n4fc3/uhgH/iCOvvo12qOj0V9sMVIXOBm9zpm4BTWkYikiIi8e50GnAhbh9fYRbM8fvHOxL4WN07cxFUaVxlroMPw7lOHW1zgfHuk0L9gJySy5LRJCLNSq6Di0gfnPNFdsVrhWW/ArwIbFTVP5dTrdq/s2DiisZ3JiLpbksDEUkEBgNfl6kW3t/H6rz7Xxt+gBE42Tkf+B5Y4Ja3AOb71bsC56mKLTiXuCIdVypO78Dfup+N3fLewDR3+gJgHc7TROuAWyMYzynHD0zG6agSIAF4E9gMfAGcVU3//yqL64/ABvc7+gQ4pxpimgnswRnFMhO4Ffg5zgiX4FxGeNaNeR3lPM0Xhbgm+X1Xy4ALqimuATiXUdbi9FW32v3/GtXvLMi4qv07A7oB/3HjWg88GODffVh/H63LEWOMMSGxS1XGGGNCYonDGGNMSCxxGGOMCYklDmOMMSGxxGGMMSYkljiMCQMROVp5rQrXf8t92x8RqS8iz4vIFre3089EpK+IxLnTdfLFXVNzWOIwJsrc/oxiVHWrWzQN563eDqraGadH3zR1Om78CBgdlUCNcVniMCaM3DeZnxSR9SKyTkRGu+Uet/uJDSIyT0Tmi8hId7VxuD0BiEh7oC/wO1X1gdN9jKq+79Z9161vTNRYk9eY8LoG6AF0B9KAFSLyGU73L22Brjg9G28E/u6ucyHOW9wAnYHV6nRaF8h64PyIRG5MkKzFYUx4DcDpfbdYVb8HPsU50Q8A3lRVn6ruxenupERzICuYjbsJpUBEksMctzFBs8RhTHiVNzhORYPm5OH0JQROP0fdRaSi38144PhpxGZMWFjiMCa8PgNGuwPrpOMMz/oF8DnOAFIeEWmKM8RoiY3A2QDqjJ2yEnjYr5fVDiIy3J1OBbJUtbC6DsiYsixxGBNec3B6KV0DfAzc416aehunB9r1wPM4I8fluOu8z8mJ5DacAb02i8g6nHFDSsaauBSYH9lDMKZi1juuMdVEROqrMzJcKk4r5EJV3euOofCJO1/eTfGSbbwD/FZVv6mGkI0JyJ6qMqb6zHMH3IkDprgtEVQ1T0R+jzMu9I7yVnYHp3rXkoaJNmtxGGOMCYnd4zDGGBMSSxzGGGNCYonDGGNMSCxxGGOMCYklDmOMMSH5f/QK3OuUuI2cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线  可以从cv_result看到超参数变化的时候，模型性能的变化趋势\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    plt.plot(x_axis, -test_scores[:,i], label = penaltys[i] +' Test')\n",
    "    plt.plot(x_axis, -train_scores[:,i], label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_logloss.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。 可以看出在训练集上C越大（正则越少）的模型性能越好； 但在测试集上当C=1时性能最好（L1正则），模型选择时， 是要选择在测试集上最好的性能"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 评价指标为 accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='auto',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='liblinear',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'C': [0.01, 0.1, 1, 10, 100], 'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.01, 0.1, 1, 10,100]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grida= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='accuracy',n_jobs = 4, return_train_score=True)#对每一组参数做5折交叉验证，所以有5重循环\n",
    "grida.fit(X_train,y_train) ## fit是对每一组参数做一个循环，多一组，时间就线性增长"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7747644512350395\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grida.best_score_)\n",
    "print(grida.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l1 Test [-2. -1.  0.  1.  2.] [0.70706222 0.76304219 0.77216705 0.77086835 0.77086835]\n",
      "l1 Train [-2. -1.  0.  1.  2.] [0.70703318 0.76985832 0.77473955 0.77474061 0.77441487]\n",
      "l2 Test [-2. -1.  0.  1.  2.] [0.76178593 0.77476445 0.77086835 0.77086835 0.77086835]\n",
      "l2 Train [-2. -1.  0.  1.  2.] [0.76400042 0.77311353 0.77376341 0.77441487 0.77441487]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhUZZrw/++dfQ+BBAhbEvYtLLKIAqIoNgKCuDW0uyLiaL/T9ky/09Pjz7Ztp7sdx3Zm3rFFiLYoKq4oAuLO0rRgwr4peyAJgbBkXyqVen5/VCVUkkqoQJ1USN2f66qr6jznOafuKsK565zznPuIMQallFKqoSB/B6CUUqpt0gShlFLKI00QSimlPNIEoZRSyiNNEEoppTwK8XcAvpKYmGhSU1P9HYZSSl1WtmzZctoYk+RpXrtJEKmpqWRlZfk7DKWUuqyISHZT8/QQk1JKKY80QSillPJIE4RSSimPNEEopZTySBOEUkopjzRBKKWU8kgThFJKKY/azXUQqnWUVZex8tBKAFLjU0mJS6FLVBdExM+RtUHVlVCwD07ugcpif0dzeREBCYagINdzCAQFu14HgwR511Zvfm1biKuvN22uGAKUJgjllVJbKe/88A5L9i6hqKqo3rzIkEhS4lLqHqlxqaTGpZISn0JcWJyfIm5lZWfg5C7Id3sU/Aimxt+RtQpjwDjAOMT1AFMjTbS5tTfRp+0JciWtINdDAPfp2vlubbhPu57x0JemlvXwno3W62wP6dqdDr/8T59/ak0QqlmltlLe/uFt3tj7BkVVRUzsPpEFwxfQOaozR4uPkl2U7Xwuzmbvmb18mf0lDuOoW75jRMe6pOH+3CuuF2HBYX78ZBfJ4YDCo/UTQf4uKM493ye2G3RNhwHTnM9d0yGq0yW9rXE4MNXVmGq769l2/rWtupk2t4etGmNvMK+2rdqOsdUubzv/Xq42R7UNXOt2uJah9nV1Ndjtl/a9XvaM6wHQ+j8KIpL30OGXvl+vtJc7yo0ePdpoqQ3fKbWV8ta+t3hj7xsU24q5psc1PDr8UYYmDm12ueqaao6XHq+XOGqfT1ecrusXJEEkRyefTxzx5xNI1+iuBEnr7taf3wBXuzaK1ZiyEszJHzD5+zD5P2IKDmIKjmBsFa5fvcGY6GRMTA/nc1QXTEQiRsJcG2Hb+ee6jbbNtZG1gavN4WrD1d5oA2+zWbIBltDQ84+wsPrPF9N2UesJRUIbvA4LRQL4sE6LORxgHEh4xEUtLiJbjDGjPc3TPQhVT4mthLf2vcWbe9+k2FbMpB6TeHT4owxJHOLV8qHBofSO703v+N712o3DQUnpGY6fPcKxc0fIO5tN7rls8rOPs6Uoi822SkJqIKTGEEko3cKS6BqWSJfQjiSGJJAYGk+noDgiCDm/AXffmLu9rt3INt7gNtj4ui3r/QY4yvWoVQEccD3qq9sQhoQ4n5vYUAbHxPpgI93Ehja0/usg1zKEhup5I3VBmiAUAMW24rrEUGIr4dqe17Jg+AKGdDqfGIzNRunf/07JF19iP11Q/zBFtfsv5sYbbmqcu91BQKrr0bQqIMf1cIvR9ajlCA2G0BCCwsIJDgsnKNTzRjgoNrL+r2VThdiLkapCpOosUnnaOR0EEmSQqHikY3ekYy+kUwqS1Bvp0B0JD2+0kXbf4Aa5vadugFV7oAkiwBXbinlrrysxVJdwXc/rWDB8AYM7DQacv/zLs7IoXrmKks8/p6aoiKD4eMJ69jy/kYyL9P4Xblho437NbHDtwcIp+zlyKk6QXZFHdmUeR8uPc7gkm1OVBTiP95YD5SRHJ5MS1915kjw+lZSoZFJsVXQrOkHwyT3OcwUn94Ct1PnhJRiSBkDXiefPFXRJh+hLO1+gVHuhCSJAFduKWbp3KUv3LqWkuoTJPSezYPgCBnUahDGGij17KF65iuLVq7GfPIlERRE7eTJxM6YTc/XVSFjrnGAOB9Jcj4kN5pVXl5NdnE12cTZHTu8hu2AX2WcPsfJEJqVuJwpDjaGX3UFKSDQpfUeR1nEAKcmjSek1kY4xyfpLX6km6EnqAFNUVcTSfc7EUFpdyvW9rmfB8AUM7DiQqsNHKF61iuJVq7AdPQqhocRMnEjc9GnEXncdQVFRF1y/5RwOOHsY8nfCyd3nRxGVnKjrYuJ6cKbLILI7dic7Kp6jwcLRqnNkl2RzrOQYdsf58w2xYbF1J8rrhujGp9IrthdRoZf2eQ8VlLJiex6rd52gqKKamPAQosKDiQ4LISY8hOjwEKJd09HhIXXzY8JD6tqiw4Pr5kWHhxAVGkxQkCY05TvNnaTWBBEgiqqKeHPvm7y17y1Kq0u5odcNLBi+gN62eIpXf0bxypVU7t0LIkSNHUvcjOnETZlCcIcO/gvaVg6n9jmTQW0iOLkHqsuc84NCIGmg69DQULchpR2bXKXdYedE2QmOFtUfYXW0+Cj5Zfn1+naO6nz+mg7XSKvUuFS6xXQjJMjzzveJogpW7jjBJzty2Z1bjAhcmdaRlI7RlNnslFXZKauqobTKTrnNTmlVDWVVdiqqvR8aGRV2Pmm4v44ODyHaNe1sCyaqmWQU7ZofrAknoGmCCGBFVUUs2bOEt394m7LqMqakTGF+yly6bD5M8apVlGdlgTFEpKcTN30acTdNI7RL59YPtPRU42sLzhxwXn0FEB53PgHUPpIGQki4z0KosFdwrPhYXdKoTRxHi45SbDt/ejxEQugR26NubyMpvDu5p2PI3B/MtqM1GCMM6xHPzOHdmDGsG13jLzz8sMZhKLedTx5lVXZXQnEmkIYJxTn//LyyKjvltpp6r70VGRrsIaG4Ek3Y+WQT3SDR1Es8dQlKE87lRhNEACqsLOSNvW/UJYabulzHg4XpRH+7hdKNG8FuJ6x3b+JmTCd+2jTCWut+3o6a84eI6pLBbih1+/Ue3wu6Dq2fDDqkuK5ebX3GGAqrCuvtcRw8d5h9pw9zqiIHI+cPWYWK86ryfh3T6l9RHtq6V5Q7jKGiuoYKWw1lNjsVthrKbc7p8mr7+de2Gipsdsqrz0+X2+xU2BxU2JxJyDnfceE3dQkPCSI6LJjI8GCiQkOIDAsmKizY+RwaQmRYEFFhIa62EKJCg4gMc+79RIaFEBUWRGiwXgfRElFh4QzrmnpRy/rtOggRmQr8NxAMZBhj/tRg/ovAda7JKKCzMaaDiFwHvOjWdSAwxxjzsZXxtgfnKs85E8O+t7FVlfNA6QhuPBAJf/sbpuJLKpOT6XjfvcTPmEH4wIHWnqC1lcOpvR4OEZU75weFQNIg6DPZbRTRkGYPEfmDiJAQkUB0SDynTydz8McUvto7gIrqGrrGh3H90HCGpFRBaEHdnsfOgp2sObIGw2X2A0xwjgxw2zELAmJauBqb61GvKIvd9ai4lACVJ+E1aWQ9uMLn67UsQYhIMPASMAXngPZMEVlhjNlb28cY84Rb/58DI13t3wIjXO0dgYPAF1bF2h6cqzzHkj1LWLb3bdKOlPOvx7oxYIcDSrIISkgg9pZZxE+fTuQVV1hzlWrJyca1iM4cdDtEFO9MAFfc53aIaIBPDxFZocZh+P7IWVbsyGX1rnyKKqrpEBXKrVd0Z+bwboxJ7djkSeOqmiqOFx8nuzibcnt5K0fethljsNU4qLI7qKp2UGWvqXuutDtwOC6zxOpnnaOt+VFl5R7EWOCgMeYwgIgsA2YBe5voPxf4rYf224HPjDH6P8yDs5VnWbL7dTZ9+xajd1by0v5QoopqCIo6Q+yUG4ibPp3oq65yXlvgK8UnIHtj/WRQdur8/A69oOswGHrb+RPIHXr57RBRSxlj2J1bzCfbc/l0Zx4ni6uICgvmxsFdmDmiGxP6JhEWcuEkGx4cTt+EvvRN6NsKUSvle1YmiO7AcbfpHOBKTx1FJAXnUPdvPMyeA/y5ieXmA/MBevXqdSmxXnbOVp7l/S9e5OynK7hyt42p54DQEGImTSB+xgxiJk0iKDLS929cfhb+ciVUFkFQKHQeCP2m1D9EFJng+/dtBbXDUlfsyOPI6TJCg4VJ/Tvz5PRuXD+oM1FhetmQCixW/sV7+rnY1H7jHOADY+rXRhaRZCAd+NzTQsaYRcAicJ6kvvhQLx+nDu9h4xv/QeS3WVxz0oFDIHj0CLrecjuxU6YQHGfxydDtbzuTw10fQNokCLkMK7K6OVFUwac7nEmhdljquLROPHJNb6YO7UqHqMv78yl1KaxMEDlAT7fpHkBeE33nAI95aL8TWG6MqfZxbJcV+9mznPj0I459uJSO+08yEChISyD4l3fSb/ZdhCQltU4gDgdkvQo9xzn3Gi5T58psrN59ghXb8/j+6FmMgeE94nly+iBuHt6NLnEXVxVTqfbGygSRCfQTkTQgF2cS+FnDTiIyAEgAvvOwjrnAv1oYY5tVU1pG6ddfcXrFx1R+t5kgh6EsUci+ZRBX3vPPXDPk6tYP6vC3ziGq1/6m9d/7EpVV2flq30k+2Z7H+v0F2B2G3knR/OL6/swc0Y20xGh/h6hUm2NZgjDG2EXkcZyHh4KB14wxe0TkGSDLGFM7JmsusMw0uCBDRFJx7oGssyrGtsZhs1G2fj1FK1dR8u03UGWjIF7YeKUQcuNkbr/pn7ixQ5r/AszMgOgkGDzTfzG0gM3uYP3+Aj7ZkcdXe09SUV1DcnwED01I4+bh3RjSLU7rMCnVDEvPuhljVgOrG7Q91WD66SaWPYrzRHe7ZmpqKN+8maJVqyj54kscJSVUxUawPt3BhsGh9J84g/nDHyElLsW/gRYeg/1rYMITbXpoqqdhqQleDktVStWnwzL8wBhD5c6dFK1cRfGaz6gpOI1ER5E9MpllPW3sSHEwre8sXhg2n15xbWR01pbXnc+j7vdnFB4ZY9iVW8SK7XmNhqXOGtGdCf0S9cpcpS6CJohWVHXgAEWrVlG8ajXVx48jYWGEThjHd0OCWBj1PRXBOdzc52aeSZ9Pz7ieF15ha7FXwdY3oP9U5/UMbYQOS1XKWvo/yGK2nFyKV6+meOVKqvbvh6Agoq+6irCH7uLdLkdZlrOCGlPDzD4zeTj94baVGGrtXQFlBTDmIX9H4nFY6lW9ncNSbxqaTHyUDy8IVCrAaYKwgP3MGYo/W0PxqlVUbNsGQOTIkXR58kmqrrmCv55Yzof7/wfHcQcz+85kXvo8esa2wcRQKzMDOvaG3pP98va1w1I/2Z5Hpg5LVarVaILwkZrSUkq+/IrilSsp27QJamoI79+fpF/+krhp0zibEMzLu17lw3UvYIxhVt9ZzEufR4/YHv4OvXn5u+D4Jrjx38GKGk5NaGpY6hM39Ofm4TosVanWoAniEjiqqihdu47iVasoXbsWY7MR2qMHnebNI276NCL69ye/LJ//2JXBR99+hDGGW/rdwrz0eXSPuUwGaGW+CiERMKLRJSw+p8NSlWpbNEG0kLHbKdu0meKVKyn56iscpaUEJybS4ac/JX76NCKGD0dEOFF6glc3PctHBz7CYJjddzbz0ufRLaabvz+C9yqLYOd7MPR2y0pw1zgMm4+c4dMdeY2Gpc4a0Z3RKQk6LFUpP9EE4QVjDBXbtjvv17xmDTVnzhAUG0vsjTcSP2M6UWPHIiHOrzKvNI+MXRksP7gcgFv73sq89HkkxyT78yNcnB3LnLf3HDvPp6vVYalKXR40QTSj8sf9FK9cSfHq1VTn5iLh4cRcdx1x06cRc801BIWfv2AsrzSPxbsW8/FB5z2Nbut3Gw8NfejyTAwAxjhPTncfBd1G+mSVB0+VsmJHHp+6DUu9dkBnZg7vxg2DuhAZFuyT91FK+YYmiAZsx49TvGo1xatWUnXgIAQHEz3+apL+z8+Juf56gmPq31srtzSXxTsX88mhTxCE2/rdxrz0eXSN7uqnT+AjRzfA6f1wy8uXtJraYamfbM9jT975YakLJvVm6hAdlqpUW6YJArAXFFD82RqKVq2kcsdOACJHjaLrb58i9ic/IaRj4+PvOSU5ZOzK4JODnyAi3N7vdh5Kf+jyTwy1vl/svK/DkNktXrSpYan/34zBzBiWrMNSlbpMBHyCsB07xqGpN4HDQfigQXT+538ibto0Qrt5Ppl8vOQ4GbsyWHFwBUESxB0D7uDBoQ+2n8QAUJwHP6yCqx6DUO9uOuRpWGof17DUmcO7karDUpW67AR8ggjt2ZPOv/oVMRMnEN636VtDHi85zuKdi1lxaAXBEsydA+7kwaEP0iW6SytG20q2LHHeS3r0A812s9kdrNtfwAq3YandXMNSZ47oxuBkHZaq1OUs4BOEiNDpgfubnH+8+DiLdi3i00OfEhIUwtyBc3lg6AN0jurcekG2pppqZ2G+vjc4r572YGdOIe98f0yHpSrVzgV8gmjKseJjLNq5iJWHV9YlhgeHPkhSVCvdvc1fflgFpfkw5r89zs4trOC2l/9OaHAQPxnSlZnDu+mwVKXaKU0QDWQXZ7No5yJWHV5FSFAIPxv0Mx4Y8kD7Twy1MjOcFVubuKXoqxuOYAx8/otr6NkxqpWDU0q1Jk0QLkeLjjoTw5FVhAWFcdegu3hg6AMkRib6O7TWc+oH5/DWG56GoMbXJJwrs/HO98eYObybJgelAkDAJ4jTFaf5c9af6xLDPYPu4f6h9wdWYqiV9SoEh8HIezzOfuO7bCqqa3hkUp9WDkwp5Q8BnyAigiP4Pv977h18L/cNuS8wEwNAVSlsf8d53UN04++gwlbDku+OMnlgZwZ0jW39+JRSrS7gE0RMWAyf3fYZoUEBfkXvznfBVgJjHvY4+72s45wts/Hotbr3oFSgsHToiYhMFZEfReSgiPzaw/wXRWS767FfRArd5vUSkS9EZJ+I7BWRVKviDPjkYIyzrHfXYdBjdKPZ1TUOFq0/zKiUBMakWlPVVSnV9li2ByEiwcBLwBQgB8gUkRXGmL21fYwxT7j1/zngXhXuDeDfjTFfikgM4LAq1oB3bBOc2gM3/w94uLBt1c4T5BZW8PTMIX4ITinlL1buQYwFDhpjDhtjbMAyYFYz/ecC7wCIyGAgxBjzJYAxptQYU25hrIEtMwPC4yH99kazjDEsXHeIfp1juH5gO704UCnlkZUJojtw3G06x9XWiIikAGnAN66m/kChiHwkIttE5HnXHknD5eaLSJaIZBUUFPg4/ABRegr2fuK8Y1xY43pJa/cX8EN+CY9M6qNXSCsVYKxMEJ62JqaJvnOAD4wxNa7pEGAi8M/AGKA3cH+jlRmzyBgz2hgzOikpQC5k87WtS8BRDWMe8jh74dpDJMdHMHP4ZXQnPKWUT1iZIHKAnm7TPYC8JvrOwXV4yW3Zba7DU3bgY+AKS6IMZDV2yHodel8Lif0azd567Bybj5zloQlphIVoKQ2lAo2V/+szgX4ikiYiYTiTwIqGnURkAJAAfNdg2QQRqd0tmAzsbbisukQHPofiHBjj+ZaiC9ceIj4ylLlje7VyYEqptsCyBOH65f848DmwD3jPGLNHRJ4RkZluXecCy4wxxm3ZGpyHl74WkV04D1cttirWgPX9YojrDv1vajTr4KlSvtx3knuvSiE6POAvl1EqIFn6P98YsxpY3aDtqQbTTzex7JfAMMuCC3SnD8Lhb+G6JyG48Z/BovWHCA8J4v6rU1s/NqVUm6AHlgNV1msQFAJX3Nto1omiCpZvy+XO0T3pFBPuh+CUUm2BJohAZCuH7Uth0EyIbXxHvNf+dgSHgYcner5hkFIqMGiCCES7P4DKIo8np4vKq3l78zGmpydrSW+lApwmiEBjjPPkdOfBkHJ1o9lLN2dTZqthgZb0VirgaYIINLlbIH+n88K4BnWXKqtr+OvGI0zqn8TgbnF+ClAp1VZoggg0mRkQFgPDftpo1vtbcjhdatO9B6UUoAkisJSdgd0fwfA5EF7/pj/2GgeL1x9meM8OjOutJb2VUpogAsu2N6GmyuPJ6c9253PsbDmPTuqDeCj5rZQKPJogAoWjxnntQ8oE6Dyo3qzakt69k6K5cXDjYa9KqcCkCSJQHPwaCrM9Vm3dcOA0e/KKeeSa3lrSWylVRxNEoMjMgJguMHBGo1kL1x2iS1w4t4z0eLsOpVSA0gQRCM4dhQNfwBX3QUhYvVk7cwr5+6EzPDg+jfCQRvdkUkoFME0QgSDrNZAgGHV/o1kL1x0iNiKEn12pJb2VUvVpgmjvqith65swcBrE1z+EdLiglM9253PPuBRiI0L9FKBSqq3SBNHe7f0YKs56HNq6eMNhQoODeGB8mh8CU0q1dZog2rvvF0OnfpA2qV7zqeJKPtySy+2jepAUqyW9lVKNaYJoz/K2QW6Wc++hwcVvr208it3hYL6W9FZKNUETRHuW+SqERjlLa7gprqzmrU3Z3JSeTGpitJ+CU0q1dZog2quKc7DrA0i/AyI71Jv11qZjlFTZeVSL8imlmqEJor3a/jbYKxqdnK6sruG1jUeY0DeRod3j/RScUupyYGmCEJGpIvKjiBwUkV97mP+iiGx3PfaLSKHbvBq3eSusjLPdcTich5d6XgnJw+rNWr4tl4KSKh69VvcelFLNC7FqxSISDLwETAFygEwRWWGM2VvbxxjzhFv/nwMj3VZRYYwZYVV87dqRtXD2EFxbPyfXOAyL1h8mvXs8V/fp5J/YlFKXDSv3IMYCB40xh40xNmAZMKuZ/nOBdyyMJ3BkvgpRnWBw/a/78z35HDldxgIt6a2U8oKVCaI7cNxtOsfV1oiIpABpwDduzREikiUim0TkFuvCbGcKj8OPq+GKeyHk/PUNtSW9UztFMXVoVz8GqJS6XFh2iAnw9BPVNNF3DvCBMabGra2XMSZPRHoD34jILmPMoXpvIDIfmA/Qq5fWEgJgy+tgDIx+sF7zd4fOsDOniH+fPZRgLemtlPKClXsQOUBPt+keQF4TfefQ4PCSMSbP9XwYWEv98xO1fRYZY0YbY0YnJSX5IubLm90GW5dA/6nQoX7CfHndIRJjwrntih5+Ck4pdbmxMkFkAv1EJE1EwnAmgUajkURkAJAAfOfWliAi4a7XicB4YG/DZVUD+1ZAWUGjoa27c4vYcOA0D05IJSJUS3orpbxj2SEmY4xdRB4HPgeCgdeMMXtE5BkgyxhTmyzmAsuMMe6HnwYBr4iIA2cS+5P76CfVhMwMSEiFPpPrNS9cd4iY8BDuujLFP3EppS5LVp6DwBizGljdoO2pBtNPe1ju70C6lbG1O/m74dh3MOX3EHR+xzD7TBmrd53g4Ym9iY/Ukt5KKe/pldTtRdarEBIBI++u17x4w2FCgoJ4cIKW9FZKtYxXCUJEPhSR6SKiCaUtqiyGHe/C0NsgqmNdc0FJFe9n5XDrFd3pEhfhxwCVUpcjbzf4LwM/Aw6IyJ9EZKCFMamW2vkuVJfBmIfqNb/+9yPYahzMv0ZLeiulWs6rBGGM+coYcxdwBXAU+FJE/i4iD4iIHtj2J2OcNwXqNhK6j6prLq2y8+Z32fxkcFd6J8X4MUCl1OXK60NGItIJuB+YB2wD/htnwvjSksiUd47+DU7/CGMertf8zuZjFFfaWaBF+ZRSF8mrUUwi8hEwEHgTuNkYc8I1610RybIqOOWFzAyI6ABDb61rstkdvPq3I1zVuxMjenZoZmGllGqat8Nc/9cY842nGcaY0T6MR7VE8Qn4YSVcuQBCI+uaP96eS35xJc/dPqyZhZVSqnneHmIaJCJ1P0VdVzr/g0UxKW9tXQIOe726Sw6Hsyjf4OQ4rumX6MfglFKXO28TxMPGmLqb+RhjzgEPN9NfWa2m2lmYr+8N0On8eYYv953kcEEZj0zqrSW9lVKXxNsEESRuWxvXzYDCrAlJeeXH1VByol7dpdqS3j07RjI9PdmPwSml2gNvE8TnwHsicr2ITMZZeXWNdWGpC8rMgPie0O/Guqbvj5xl27FC5k/sTUiwXtOolLo03p6k/hfgEeBRnPd5+ALIsCoodQEFP8KR9XD9UxB0vjrry+sO0Sk6jDtG92xmYaWU8o5XCcIY48B5NfXL1oajvJL5KgSHwch765r2nShm7Y8F/NOU/lrSWynlE95eB9EP+CMwGKgr6mOM0RoOra2qFHa8A4NvgZjzN0l6Zd0hosOCufeqVP/FppRqV7w9UP1XnHsPduA64A2cF82p1rbrfagqrndy+vjZcj7deYK5Y3sRH6WVT5RSvuFtgog0xnwNiDEm23UPh8kXWEb5mjHOk9Nd0qHn2LrmjA2HCRJ4aKKW9FZK+Y63J6krXaW+D7juEpcLdLYuLOXR8c1wcjfM+C9wjTo+U1rFu1nHmTWiO8nxkRdYgVJKec/bPYhfAFHA/wFGAXcD91kVlGpCZgaEx8GwO+ualnyXTWW1gwWT9HSQUsq3LrgH4boo7k5jzK+AUuABy6NSjZUWwJ6Pnfd8CIsGoNxm543vjjJlcBf6do71b3xKqXbngnsQxpgaYJRo3Qb/2roEHNUw+vxNgZZ9f5zC8moWTNKS3kop3/P2HMQ24BMReR8oq200xnxkSVSqPkcNZP0V0q6BpP4AVNc4yNhwmLGpHRmVkuDnAJVS7ZG35yA6Amdwjly62fWYcaGFRGSqiPwoIgdF5Nce5r8oIttdj/0iUthgfpyI5IrI/3oZZ/u0/3Mozql3U6AV2/PIK6pkwbV67kEpZQ1vr6Ru8XkH17mLl4ApQA6QKSIrjDF73db7hFv/nwMjG6zm98C6lr53u5OZAbHJMGAa4Czp/cr6QwzoEst1A3QwmVLKGt5eSf1XwDRsN8Y86KF7rbHAQWPMYdc6lgGzgL1N9J8L/NbtPUcBXXAWBQzcmxKdOQSHvoZrfwPBzn+ub388xf6Tpbz40+Fa0lspZRlvz0GsdHsdAcwG8i6wTHfguNt0DnClp44ikgKkAd+4poOAF4B7gOubegMRmQ/MB+jVq9cFwrlMZb0GQSEw6vyo4pfXHqJ7h0hmDOvmx8CUUu2dt4eYPnSfFpF3gK8usJinn7aN9kJc5gAfuEZMAfwDsNoYc7y5X8jGmEXAIoDRo0c3te7Ll60cti2FQTdDbFcAso6eJSv7HL+9eTChWtJbKWUhb/cgGuoHXOgnew7gXuWNwEgAABzpSURBVHe6B03vdcwBHnObvgqY6LqtaQwQJiKlxphGJ7rbtT0fQWVhvbpLC9cdIiEqlJ+O0ZLeSilreXsOooT6v/7zcd4jojmZQD8RScNZmmMO8DMP6x4AJADf1bYZY+5ym38/MDrgkgM4T04nDYSU8QDsP1nCV/tO8Ysb+hEVdrG5XSmlvOPtIaYWX6ZrjLG76jZ9DgQDrxlj9ojIM0CWMWaFq+tcYJkxpv0dIroUOVsgbxtM+8+6ukuvrDtMZGgw92lJb6VUK/B2D2I28I0xpsg13QG41hjzcXPLGWNWA6sbtD3VYPrpC6zjdeB1b+JsVzIzICwGhv0UgNzCCj7Znsvd41JIiNbbgSulrOftWc7f1iYHAGNMIW5DUpWPlZ+F3R86k0NEHACvbjiCAeZpSW+lVCvxNkF46qcHwa2ybSnUVDkL8wGF5TaWZR5j1vBu9EiI8nNwSqlA4W2CyBKRP4tIHxHpLSIvAlusDCxgORyQ9Sr0uhq6DAHgje+yKbfV8IgW5VNKtSJvE8TPARvwLvAeUEH9YanKVw59DeeOwljn0NYKWw2v//0okwd2ZkBXLemtlGo93o5iKgMCb5ipP2RmQHRnGHgzAO9lHedsmU1LeiulWp1XexAi8qVr5FLtdIKIfG5dWAHqXLazcuuo+yAkDHuNg8UbDjMqJYExqVrSWynVurw9xJToGrkEgDHmHHpPat/Les15zcOo+wFYtesEOecqWDCpjxblU0q1Om8ThENE6kpriEgqTddVUhejuhK2veks6R3fA2MML689RL/OMVw/UHOxUqr1eTtU9d+Av4lI7b0ZrsFVRVX5yN5PoPxMXd2ltfsL+CG/hOdvH0ZQkO49KKVan7cnqdeIyGicSWE78AnOkUzKVzIzoFNfSJsEwMK1h0iOj2DWiO5+DkwpFai8LbUxD/hHnBVZtwPjcBbXm2xdaAHkxA7I+R5+8kcICmLbsXNsPnKWJ6cPIixES3orpfzD263PPwJjgGxjzHU4bw1aYFlUgSYzA0IiYcRcwFnSOz4ylLlj2+lNkJRSlwVvE0SlMaYSQETCjTE/AAOsCyuAVBTCzvdh2B0QmcDBU6V8sfck916VQnS4VjNRSvmPt1ugHNd1EB8DX4rIOS58y1HljR3vgL0CRjvrLi1af4iw4CDuuzrVv3EppQKetyepZ7tePi0i3wLxwBrLogoUDofz8FKPMdBtBPlFlSzflsvcsb1IjAn3d3RKqQDX4mMYxph1F+6lvHJkHZw5CLNfAeC1jUdwGHh4Ym8/B6aUUt6fg1BWyMyAqE4w+BaKyqt5a1M209OT6dlRS3orpfxPE4S/FOXCj6th5D0QGsHSzdmU2Wp4ZJLuPSil2gZNEP6y5XUwBkY/QGV1DX/deIRJ/ZMY0i3e35EppRSgCcI/7DZnguh3IySk8sGWHE6XaklvpVTbYmmCEJGpIvKjiBwUkUb3kxCRF0Vku+uxX0QKXe0pIrLF1b5HRBZYGWer++FTKDsFYx/GXuNg0frDDO/ZgXG9O/o7MqWUqmPZlVgiEgy8BEwBcoBMEVlhjNlb28cY84Rb/5/jvEIb4ARwtTGmSkRigN2uZdvHtReZr0KHFOhzPZ/tyufY2XJ+M22glvRWSrUpVu5BjAUOGmMOG2NswDJgVjP95wLvABhjbMaYKld7uMVxtq6TeyF7I4x5CCPCwnWH6J0YzZTBXf0dmVJK1WPlhrc7cNxtOsfV1oiIpABpwDdubT1FZKdrHc952nsQkfkikiUiWQUFl0lpqMwMCA6HEXfzt4On2ZNXzCOTehOsJb2VUm2MlQnC0xavqZsMzQE+MMbU1HU05rgxZhjQF7hPRLo0Wpkxi4wxo40xo5OSknwStKUqi2HnuzD0VojuxMJ1h+gSF84tI7Wkt1Kq7bEyQeQAPd2me9B0/aY5uA4vNeTac9gDTPRpdP6w812wlcKYh9mZU8jGg2d4cHwa4SHB/o5MKaUasTJBZAL9RCRNRMJwJoEVDTuJyAAgAef9JWrbeohIpOt1AjAe+NHCWK1njPPkdPII6H4FC9cdIjYihJ9dqSW9lVJtk2UJwhhjBx4HPgf2Ae8ZY/aIyDMiMtOt61xgmTHG/fDTIGCziOwA1gH/aYzZZVWsrSJ7IxTsgzHzOHKmnM9253PPuBRiI0L9HZlSSnlk6Q0HjDGrgdUN2p5qMP20h+W+BIZZGVury8yAiA4w9DYWrTxEaHAQD4xP83dUSinVpPYzfLQtK8mHfZ/CyLs5VRnEh1tyuH1UD5JitaS3Uqrt0gTRGra+AQ47jH6Q1zYexe5wMF9Leiul2jhNEFarsUPWX6HPZIqje/HWpmxuSk8mNTHa35EppVSz9KbHVvtxNZTkwfT/5O3NxyipsvOoFuVTyqeqq6vJycmhsrLS36G0WREREfTo0YPQUO8HxmiCsFpmBsT3pDJtCq9+uJ4JfRMZ2l1LeivlSzk5OcTGxpKamqo1zTwwxnDmzBlycnJIS/N+cIweYrJSwX7nbUVH3c/yHfkUlFRpSW+lLFBZWUmnTp00OTRBROjUqVOL97A0QVgp61UICqVmxD0sWn+Yod3jGN+3k7+jUqpd0uTQvIv5fjRBWMVWBtvfhsGz+CLbwZHTZTw6qa/+ESulLhuaIKyy632oKsaMmcfCdYdI7RTF1KFa0lup9iomJqbu9dSpU+nQoQMzZszw2Pexxx5jxIgRDB48mMjISEaMGMGIESP44IMPWvSeW7duZc2aNZcUd3P0JLUVjHGenO4ylO+q+7Ij53v+ffZQLemtVID41a9+RXl5Oa+88orH+S+99BIAR48eZcaMGWzfvv2i3mfr1q3s3r2bqVOnXnSszdEEYYXj30P+LpjxIi+vO0xiTDi3XdHD31EpFRB+9+ke9uYV+3Sdg7vF8dubh3jd//rrr2ft2rUX9V4HDhzg8ccf5/Tp00RHR5ORkUH//v1ZtmwZzz77LMHBwXTs2JHVq1fzzDPPUFFRwdq1a3nyySe5/fbbL+o9m6IJwgqZGRAWy97EqWw4sI3/O3UAEaFa0lspdWHz588nIyODPn36sHHjRh5//HG++OILfve737F27Vq6dOlCYWEhkZGRPPXUU+zevZv/+q//siQWTRC+VloAez+GUffz8ncniQkP4a4rU/wdlVIBoyW/9NuawsJCNm3axG233VbXZrfbARg/fjz33nsvd9xxB7feemurxKMJwte2vQk1NvL63cWq13J5eGJv4iO1pLdS6sKMMSQmJno8J7F48WI2b97MypUrGT58ODt37rQ8Hh3F5EuOGmfdpdSJ/GVPMCFBQTw4QUt6K6W8k5CQQHJyMsuXLwfA4XCwY8cOAA4fPsy4ceP4/e9/T0JCArm5ucTGxlJSUmJZPJogfOnAF1B0jOL0+3k/K4fZI7vTJS7C31EppVrZxIkTueOOO/j666/p0aMHn3/+udfLLlu2jIULFzJ8+HCGDBnCypUrAXjiiSdIT08nPT2dG264gaFDhzJ58mR27NjByJEjWzxE1ht6iMmXMjMgNpmMU4Ow1Rxl/iQt6a1UoCgtLa17vWHDBq+WSU1NZffu3fXaevfu7TGhrFjR6I7NJCUlkZWV1cJIvad7EL5y9jAc/Iqq4ffy+uYcfjK4K32SYi68nFJKtVGaIHwl6zWQYD4wkymutLPgWi3Kp5S6vGmC8IXqCti2FMfAGfy/zDLG9e7IiJ4d/B2VUkpdEksThIhMFZEfReSgiPzaw/wXRWS767FfRApd7SNE5DsR2SMiO0Xkp1bGecl2fwQV51gXP4v84koevbavvyNSSqlLZtlJahEJBl4CpgA5QKaIrDDG7K3tY4x5wq3/z4GRrsly4F5jzAER6QZsEZHPjTGFVsV7STIzMIkDeHZ3RwYnh3BNv0R/R6SUUpfMyj2IscBBY8xhY4wNWAbMaqb/XOAdAGPMfmPMAdfrPOAUkGRhrBcvdwvkbeWHnndy6HQ5j0zqrSW9lVLtgpUJojtw3G06x9XWiIikAGnANx7mjQXCgEMe5s0XkSwRySooKPBJ0C2W+SomNJpnjw+jZ8dIpqcn+ycOpZRftXa57+XLl/P8889fctzNsfI6CE8/o00TfecAHxhjauqtQCQZeBO4zxjjaLQyYxYBiwBGjx7d1LqtU34Wdn/Iqd6z2bizmt/P6k9IsJ73VyrQ+arct91uJyTE82Z69uzZvgm2GVYmiBygp9t0DyCvib5zgMfcG0QkDlgFPGmM2WRJhJdq+1tgr+R/S66lU3QYd4zueeFllFLW+uzXznL7vtQ1HW76k9fdL6Xc94QJE5g0aRIbNmzg1ltvJS0tjT/84Q/YbDaSkpJYunQpnTt3JiMjo66S6913302nTp3IzMwkPz+fF154wScJxMoEkQn0E5E0IBdnEvhZw04iMgBIAL5zawsDlgNvGGPetzDGi+dwQOarlHcdw5tHYvmnKala0lsp5RPFxcWsX78egHPnzjFz5kxEhIULF/LCCy/w3HPPNVrm1KlTbNy4kV27dnHnnXe27QRhjLGLyOPA50Aw8JoxZo+IPANkGWNqrxufCywzxrgfIroTuAboJCL3u9ruN8Zc3G2XrHDoGzh3hA+63U1UWDD3XKUlvZVqE1rwS7+tmjNnTt3rY8eOceedd5Kfn09VVRX9+/f3uMwtt9yCiDBs2DByc3N9EoeltZiMMauB1Q3anmow/bSH5ZYCS62M7ZJlZlATmcgfj/bjZ1f3okNUmL8jUkq1E9HR0XWvH3vsMX7zm98wbdo0vvrqK/70J88JMDw8vO51/d/bF0/PqF6Mc9mwfw0b46Zhl1DmTdSS3kopaxQVFdG9e3eMMSxZsqRV31sTxMXY8jpGhN/mjWXWiO4kx0f6OyKlVBtyKeW+G3r66aeZPXs2kyZNokuXLj6M8sLEV7si/jZ69GhjZdnbOvYq+PNgDkYM4Ya8R/jql9fQt3Os9e+rlGrSvn37GDRokL/DaPM8fU8issUYM9pTf70fREvt/QTKT/NCxURuGNRFk4NSqt3SBNFSmRkURfZkzbmBfKAlvZVS7Zieg2iJEzvh+GaWVN/AmNRERqUk+DsipZSyjCaIlsh6FXtwBBmlV7HgWr2dqFKqfdME4a2KQszO9/gqaALJXZK5bkBnf0eklFKW0gThrR3LkOpy/rf0WhZcqyW9lVLtnyYIbxgDmRnsDx3AufghzBjWzd8RKaXamNpy39u3b+eqq65iyJAhDBs2jHfffbdRX1+U+wbYunUra9as8Un8nugoJm8cWQ9nDrDQtoB509MI1ZLeSqkmREVF8cYbb9CvXz/y8vIYNWoUP/nJT+jQ4fx96r0t930hW7duZffu3UydOtUnsTekCcIbmYspDYplY/hEnh2jJb2Vasue+/45fjj7g0/XObDjQP5l7L941de9mF63bt3o3LkzBQUF9RJEcw4cOMDjjz/O6dOniY6OJiMjg/79+7Ns2TKeffZZgoOD6dixI6tXr+aZZ56hoqKCtWvX8uSTT3L77bdf1OdriiaICynKxfywmqXVNzFnUn+iwvQrU0p55/vvv8dms9Gnj/fXTM2fP5+MjAz69OnDxo0befzxx/niiy/43e9+x9q1a+nSpQuFhYVERkby1FNP1d0Twgq6tbuQrUswxsGHciPvXp3q72iUUhfg7S99q504cYJ77rmHJUuWEBTk3WHpwsJCNm3axG233VbXZrfbARg/fjz33nsvd9xxB7feeqslMTekCaI5NdXUZP6V9Y7hjB8zmo7RWtJbKXVhxcXFTJ8+nWeffZZx48Z5vZwxhsTERI/nJBYvXszmzZtZuXIlw4cPZ+fOnb4M2SM929qcfZ8SXH6KpTVTtKS3UsorNpuN2bNn1/3ab4mEhASSk5NZvnw5AA6Hgx07dgBw+PBhxo0bx+9//3sSEhLIzc0lNjaWkpISn3+GWpogmmHfvJgck0R8+k30SIjydzhKqcvAe++9x/r163n99dfrhq+2ZJTSsmXLWLhwIcOHD2fIkCGsXLkSgCeeeIL09HTS09O54YYbGDp0KJMnT2bHjh2MHDmyxUNkvaHlvptyah/8ZRx/rJ7L7J//BwO7xvlu3Uopn9Jy397Rct8+Yt+cQQ2hnOh9uyYHpVRA0kNMnlSV4Nj+DitrruTuyVf4OxqllPILSxOEiEwVkR9F5KCI/NrD/BdFZLvrsV9ECt3mrRGRQhFZaWWMntRsX0ZYTRmZibcyJlVLeiulApNlh5hEJBh4CZgC5ACZIrLCGLO3to8x5gm3/j8HRrqt4nkgCnjEqhg9Moayja9wzJHK5OunaVE+pVTAsnIPYixw0Bhz2BhjA5YBs5rpPxd4p3bCGPM1YN34rSaY7L8TV3yANVEzuGFw19Z+e6WUajOsTBDdgeNu0zmutkZEJAVIA75pyRuIyHwRyRKRrIKCgosO1F3BNy9RbKLofd19BAXp3oNSKnBZmSA8bV2bGlM7B/jAGFPTkjcwxiwyxow2xoxOSkpqcYCNlJyk47E1rA6ZzIxRfS99fUqpgNHa5b6XL1/O888/77P4PbFymGsO4F76tAeQ10TfOcBjFsbilbxvX6EbNQSNnUdYiA7wUkq1nC/LfdvtdkJCPG+mZ8+e7fvgG7AyQWQC/UQkDcjFmQR+1rCTiAwAEoDvLIzlwmrsROx4g+8YxrRrJ/o1FKXUxcv/wx+o2ufbct/hgwbS9Te/8arvpZb7njBhApMmTWLDhg3ceuutpKWl8Yc//AGbzUZSUhJLly6lc+fOZGRk1FVyvfvuu+nUqROZmZnk5+fzwgsv+CSBWJYgjDF2EXkc+BwIBl4zxuwRkWeALGPMClfXucAy0+CSbhHZAAwEYkQkB3jIGPO5VfHmZy2na00Bpwb8iphwvX5QKXXpLqbcNziL/a1fvx6Ac+fOMXPmTESEhQsX8sILL/Dcc881WubUqVNs3LiRXbt2ceedd7btBAFgjFkNrG7Q9lSD6aebWLZVf8aXrF+IMR0ZP/3u1nxbpZSPeftL32oXU+671pw5c+peHzt2jDvvvJP8/Hyqqqrq7aG4u+WWWxARhg0bRm5u7iXFXksPtAMFR3bTryyLPcm3kRgX7e9wlFKXuYst910rOvr8duixxx7jiSeeYNeuXfzlL3+hsrLS4zLh4eF1r31VY08TBHBkzf/DZoIZNP1xf4eilLrMXUq5b0+Kioro3r07xhiWLFnigwi9F/AJoqiokIH5K9gVN4nuPVP9HY5S6jJ3qeW+G3r66aeZPXs2kyZNokuXLj6M9MICvtz32fxscpf9gphr/oG0K6ZYEJlSympa7ts7Wu67hTp2TaHjL5b7OwyllGpzAv4Qk1JKKc80QSil2oX2crjcKhfz/WiCUEpd9iIiIjhz5owmiSYYYzhz5gwREREtWi7gz0EopS5/PXr0ICcnB19VdW6PIiIi6NGjR4uW0QShlLrshYaGkpaW5u8w2h09xKSUUsojTRBKKaU80gShlFLKo3ZzJbWIFADZl7CKROC0j8LxJY2rZTSultG4WqY9xpVijPF4S852kyAulYhkNXW5uT9pXC2jcbWMxtUygRaXHmJSSinlkSYIpZRSHmmCOG+RvwNogsbVMhpXy2hcLRNQcek5CKWUUh7pHoRSSimPNEEopZTyKGAThIg8LyI/iMhOEVkuIh2a6DdVRH4UkYMi8utWiOsOEdkjIg4RaXLYmogcFZFdIrJdRFp+Kz3r4mrt76ujiHwpIgdczwlN9KtxfVfbRWSFhfE0+/lFJFxE3nXN3ywiqVbF0sK47heRArfvaF4rxPSaiJwSkd1NzBcR+R9XzDtF5AqrY/IyrmtFpMjtu3qqleLqKSLfisg+1//Ff/TQx7ffmTEmIB/AjUCI6/VzwHMe+gQDh4DeQBiwAxhscVyDgAHAWmB0M/2OAomt+H1dMC4/fV//Afza9frXnv4dXfNKW+E7uuDnB/4BWOh6PQd4t43EdT/wv6319+R6z2uAK4DdTcyfBnwGCDAO2NxG4roWWNma35XrfZOBK1yvY4H9Hv4dffqdBewehDHmC2OM3TW5CfBUB3cscNAYc9gYYwOWAbMsjmufMeZHK9/jYngZV6t/X671L3G9XgLcYvH7Ncebz+8e7wfA9SIibSCuVmeMWQ+cbabLLOAN47QJ6CAiyW0gLr8wxpwwxmx1vS4B9gHdG3Tz6XcWsAmigQdxZt2GugPH3aZzaPwP4i8G+EJEtojIfH8H4+KP76uLMeYEOP8DAZ2b6BchIlkisklErEoi3nz+uj6uHyhFQCeL4mlJXAC3uQ5LfCAiPS2OyRtt+f/fVSKyQ0Q+E5Ehrf3mrkOTI4HNDWb59Dtr1/eDEJGvgK4eZv2bMeYTV59/A+zAW55W4aHtkscFexOXF8YbY/JEpDPwpYj84Prl48+4Wv37asFqerm+r97ANyKyyxhz6FJja8Cbz2/Jd3QB3rznp8A7xpgqEVmAcy9nssVxXYg/vitvbMVZv6hURKYBHwP9WuvNRSQG+BD4hTGmuOFsD4tc9HfWrhOEMeaG5uaLyH3ADOB64zqA10AO4P5LqgeQZ3VcXq4jz/V8SkSW4zyMcEkJwgdxtfr3JSInRSTZGHPCtSt9qol11H5fh0VkLc5fX75OEN58/to+OSISAsRj/eGMC8ZljDnjNrkY53k5f7Pk7+lSuW+UjTGrReQvIpJojLG8iJ+IhOJMDm8ZYz7y0MWn31nAHmISkanAvwAzjTHlTXTLBPqJSJqIhOE8qWjZCBhviUi0iMTWvsZ5wt3jiItW5o/vawVwn+v1fUCjPR0RSRCRcNfrRGA8sNeCWLz5/O7x3g5808SPk1aNq8Fx6pk4j2/72wrgXtfInHFAUe3hRH8Ska61541EZCzO7eiZ5pfyyfsK8Cqwzxjz5ya6+fY7a+0z8W3lARzEeaxuu+tRO7KkG7Dard80nKMFDuE81GJ1XLNx/gqoAk4CnzeMC+dolB2ux562Epefvq9OwNfAAddzR1f7aCDD9fpqYJfr+9oFPGRhPI0+P/AMzh8iABHA+66/v++B3lZ/R17G9UfX39IO4FtgYCvE9A5wAqh2/W09BCwAFrjmC/CSK+ZdNDOqr5Xjetztu9oEXN1KcU3Aebhop9t2a5qV35mW2lBKKeVRwB5iUkop1TxNEEoppTzSBKGUUsojTRBKKaU80gShlFLKI00QSrWAiJRe4vIfuK7mRkRiROQVETnkqs65XkSuFJEw1+t2fSGravs0QSjVSlw1e4KNMYddTRk4r6LuZ4wZgrOiaqJxFtT7GvipXwJVykUThFIXwXWl6vMisluc9+X4qas9yFV6YY+IrBSR1SJyu2uxu3Bd6S0ifYArgSeNMQ5wlgExxqxy9f3Y1V8pv9FdWKUuzq3ACGA4kAhkish6nGU8UoF0nJVl9wGvuZYZj/MqXYAhwHZjTE0T698NjLEkcqW8pHsQSl2cCTirn9YYY04C63Bu0CcA7xtjHMaYfJxlK2olAwXerNyVOGy1NbeU8gdNEEpdnKZu8tPczX8qcNZiAmctn+Ei0tz/wXCg8iJiU8onNEEodXHWAz8VkWARScJ5m8rvgb/hvPFOkIh0wXl7ylr7gL4Axnkviizgd26VQfuJyCzX605AgTGmurU+kFINaYJQ6uIsx1lVcwfwDfB/XYeUPsRZAXQ38ArOO34VuZZZRf2EMQ/njZAOisgunPdhqK3dfx2w2tqPoFTztJqrUj4mIjHGebexTjj3KsYbY/JFJBLnOYnxzZycrl3HR8C/mjZ4f3IVOHQUk1K+t1JEOgBhwO9dexYYYypE5Lc47xF8rKmFXTf1+ViTg/I33YNQSinlkZ6DUEop5ZEmCKWUUh5pglBKKeWRJgillFIeaYJQSinl0f8PmTZw6MJDunAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线  可以从cv_result看到超参数变化的时候，模型性能的变化趋势\n",
    "test_means = grida.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grida.cv_results_[ 'std_test_score' ]\n",
    "train_means = grida.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grida.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    plt.plot(x_axis, test_scores[:,i], label = penaltys[i] +' Test')\n",
    "    print(penaltys[i] +' Test',x_axis, test_scores[:,i])\n",
    "    plt.plot(x_axis, train_scores[:,i], label = penaltys[i] +' Train')\n",
    "    print(penaltys[i] +' Train',x_axis, train_scores[:,i])\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('LogisticGridSearchCV_accuacy.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的accuracy。 \n",
    "#在训练集上, accuracy最高的为 0.77474061：{'C': 1, 'penalty': 'l1'}\n",
    "#在测试集上, accuracy最高的为 0.77476445: {'C': 0.1, 'penalty': 'l2'}\n",
    "#模型选择时， 是要选择在测试集上最好的性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(grid.best_estimator_, open(\"diabetes_L1_logloss.pkl\", 'wb'))\n",
    "pickle.dump(grida.best_estimator_, open(\"diabetes_L2_accuracy.pkl\", 'wb'))\n",
    "#训练好的模型被放在grid.best_estimator_，通过dump把它存起来"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "不同评价指标得到的模型不同，正则也不同"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
